package cc.togetherit.jcrawler.down;

import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.zip.GZIPInputStream;

public class UrlUtil {
    static final public String getUrlString(String urlString, String defaultEncoding, String[][] headers, boolean gzip, int timeout) {
        InputStream in = null;
        HttpURLConnection con = null;
        try {
            URL url = new URL(urlString);
            con = (HttpURLConnection) url.openConnection();
            con.setReadTimeout(timeout);
            con.setConnectTimeout(timeout);
            if (gzip && Math.random() < 0) {
                con.setRequestProperty("Accept-Encoding", "gzip");
            }
            if (headers != null) {
                for (int i = 0; i < headers.length; i++) {
                    if (headers[i] != null) {
                        con.setRequestProperty(headers[i][0], headers[i][1]);
                        // System.out.println("setting "+headers[i][0]+": "+headers[i][1]);
                    }
                }
            }

            long starttime = System.currentTimeMillis();
            con.connect();

            int code = con.getResponseCode();
            int length = con.getContentLength();

            // if (length>=0)
            // Log007.logBaseHttp(urlString, starttime, (int)(System.currentTimeMillis()-starttime),
            // code, length);
            // String encoding = con.getContentEncoding();
            String encoding2 = con.getHeaderField("Content-Type");
            int index;
            if (encoding2 != null) {
                if ((index = encoding2.indexOf("charset=")) > 0) {
                    encoding2 = encoding2.substring(index + "charset=".length()).replace('"', ' ').replace('\'', ' ').trim();
                } else
                    encoding2 = defaultEncoding;// null;
            }
            in = new BufferedInputStream(con.getInputStream());
            // try{
            // in = new BufferedInputStream(con.getInputStream());
            // }catch (IOException e){
            // in = new BufferedInputStream(con.getErrorStream());
            // }
            if (in == null)
                return null;
            String contentencoding = con.getHeaderField("Content-Encoding");
            if (gzip && "gzip".equals(contentencoding)) {
                System.out.println("gzipped");
                in = new GZIPInputStream(in);
            }
            ByteArrayOutputStream urlData = new ByteArrayOutputStream();
            byte[] buf2 = new byte[1024];
            int n;
            while ((n = in.read(buf2)) >= 0)
                urlData.write(buf2, 0, n);

            // if (length<0)
            // Log007.logBaseHttp(urlString, starttime, (int)(System.currentTimeMillis()-starttime),
            // code, urlData.size());

            if (encoding2 != null) {
                try {
                    return urlData.toString(encoding2);
                } catch (UnsupportedEncodingException e) {
                    // e.printStackTrace();
                    System.out.println("UnsupportedEncodingException detected: " + e.getMessage());
                    return urlData.toString();
                }
            } else {
                return urlData.toString();
            }
            // return encoding2!=null?urlData.toString(encoding2):urlData.toString();
        } catch (SocketTimeoutException e) {
            System.out.println(urlString + " timeout");
            e.printStackTrace();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
            if (con != null) {
                InputStream err = con.getErrorStream();
                if (err != null) {
                    try {
                        while (err.read() >= 0);
                    } catch (IOException e1) {
                        e1.printStackTrace();
                    } finally {
                        try {
                            err.close();
                        } catch (IOException e1) {
                            e1.printStackTrace();
                        }
                    }
                }
            }
        } finally {
            if (in != null) {
                try {
                    in.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
                in = null;
            }
        }

        return null;
    }
    
}
